﻿using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Net.Http;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using HtmlAgilityPack;

namespace Homework7
{
    class Program
    {
        //用于存储已爬取的URL
        private static ConcurrentBag<string> crawledUrls = new ConcurrentBag<string>();
        
        //用于存储找到的电话号码及其对应的URL
        private static ConcurrentDictionary<string, HashSet<string>> phoneNumbers = new ConcurrentDictionary<string, HashSet<string>>();
        
        //信号量，用于控制并发线程数
        private static SemaphoreSlim semaphore = new SemaphoreSlim(5); // 最多5个并发线程
        
        //取消令牌源，用于取消爬虫任务
        private static CancellationTokenSource cts = new CancellationTokenSource();
        
        static async Task Main(string[] args)
        {
            Console.OutputEncoding = Encoding.UTF8;
            Console.WriteLine("欢迎使用高校电话号码爬虫程序！");
            
            while (true)
            {
                Console.WriteLine("\n请输入搜索关键词（输入'exit'退出）：");
                string keyword = Console.ReadLine();
                
                if (string.IsNullOrEmpty(keyword) || keyword.ToLower() == "exit")
                    break;
                
                // 重置数据
                crawledUrls = new ConcurrentBag<string>();
                phoneNumbers = new ConcurrentDictionary<string, HashSet<string>>();
                cts = new CancellationTokenSource();
                
                Console.WriteLine($"开始搜索关键词: {keyword}");
                
                try
                {
                    await StartCrawlingAsync(keyword, cts.Token);
                    
                    DisplayResults();
                }
                catch (OperationCanceledException)
                {
                    Console.WriteLine("爬虫任务已取消。");
                }
                catch (Exception ex)
                {
                    Console.WriteLine($"发生错误: {ex.Message}");
                }
            }
            
            Console.WriteLine("程序已退出。");
        }
        
        private static async Task StartCrawlingAsync(string keyword, CancellationToken cancellationToken)
        {
            string searchUrl = $"https://www.baidu.com/s?wd={Uri.EscapeDataString(keyword + " 大学 电话号码")}";
            
            using (HttpClient client = new HttpClient())
            {
                client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36");
                
                string html = await client.GetStringAsync(searchUrl);
                
                HtmlDocument doc = new HtmlDocument();
                doc.LoadHtml(html);
                
                var linkNodes = doc.DocumentNode.SelectNodes("//div[@class='result c-container']//a[@href]") ?? 
                                doc.DocumentNode.SelectNodes("//div[contains(@class, 'result')]//a[@href]") ??
                                doc.DocumentNode.SelectNodes("//a[@href]");
                
                if (linkNodes == null)
                {
                    Console.WriteLine("未找到搜索结果链接。");
                    return;
                }
                
                List<string> urls = new List<string>();
                foreach (var linkNode in linkNodes)
                {
                    string href = linkNode.GetAttributeValue("href", "");
                    if (!string.IsNullOrEmpty(href) && href.StartsWith("http"))
                    {
                        urls.Add(href);
                    }
                }
                
                Console.WriteLine($"找到 {urls.Count} 个初始链接。");
                
                //使用任务列表存储所有爬取任务
                List<Task> tasks = new List<Task>();
                
                //开始爬取
                foreach (var url in urls)
                {
                    if (phoneNumbers.Count >= 100)
                    {
                        Console.WriteLine("已找到100个不同的电话号码。");
                        break;
                    }
                    
                    tasks.Add(CrawlUrlAsync(url, client, cancellationToken));
                }
                
                //等待所有任务完成或找到100个电话号码
                while (tasks.Count > 0)
                {
                    Task completedTask = await Task.WhenAny(tasks);
                    tasks.Remove(completedTask);
                    
                    if (phoneNumbers.Count >= 100)
                    {
                        Console.WriteLine("已找到100个不同的电话号码。");
                        cts.Cancel();
                        break;
                    }
                }
            }
        }
        
        private static async Task CrawlUrlAsync(string url, HttpClient client, CancellationToken cancellationToken)
        {
            //检查是否已爬取过此URL
            if (crawledUrls.Contains(url))
                return;
            
            await semaphore.WaitAsync(cancellationToken);
            
            try
            {
                //标记URL为已爬取
                crawledUrls.Add(url);
                Console.WriteLine($"正在爬取: {url}");
                
                string html = await client.GetStringAsync(url, cancellationToken);
                
                HtmlDocument doc = new HtmlDocument();
                doc.LoadHtml(html);
                
                string pageText = doc.DocumentNode.InnerText;
                
                //查找电话号码
                ExtractPhoneNumbers(pageText, url);
            
            }
            catch (HttpRequestException ex)
            {
                Console.WriteLine($"爬取 {url} 时出错: {ex.Message}");
            }
            catch (Exception ex) when (!(ex is OperationCanceledException))
            {
                Console.WriteLine($"处理 {url} 时出错: {ex.Message}");
            }
            finally
            {
                semaphore.Release();
            }
        }
        
        private static void ExtractPhoneNumbers(string text, string url)
        {
            //中国高校电话号码模式
            string[] patterns = {
                @"(?<!\d)(\d{3,4}[-\s]?\d{7,8}(?:\s*转\s*\d+)?)(?!\d)", 
                @"(?<!\d)(\d{3,4}[-\s]?\d{3,4}[-\s]?\d{4})(?!\d)", 
                @"(?<!\d)(\(\d{3,4}\)\s*\d{7,8}(?:\s*转\s*\d+)?)(?!\d)", 
                @"(?<!\d)(86[-\s]?\d{2,3}[-\s]?\d{7,8})(?!\d)" 
            };
            
            foreach (string pattern in patterns)
            {
                var matches = Regex.Matches(text, pattern);
                foreach (Match match in matches)
                {
                    string phoneNumber = match.Groups[1].Value.Trim();
                    
                    phoneNumbers.AddOrUpdate(
                        phoneNumber,
                        new HashSet<string> { url },
                        (key, existingUrls) => {
                            existingUrls.Add(url);
                            return existingUrls;
                        }
                    );
                    
                    //已经找到了100个不同的电话号码，取消
                    if (phoneNumbers.Count >= 100)
                    {
                        cts.Cancel();
                        break;
                    }
                }
            }
        }
        
        private static void DisplayResults()
        {
            Console.WriteLine("\n===== 爬取结果 =====");
            Console.WriteLine($"已爬取的URL数量: {crawledUrls.Count}");
            Console.WriteLine($"找到的不同电话号码数量: {phoneNumbers.Count}");
            
            Console.WriteLine("\n电话号码列表:");
            int count = 0;
            foreach (var entry in phoneNumbers)
            {
                count++;
                Console.WriteLine($"{count}. 电话: {entry.Key}");
                Console.WriteLine($"   来源URL: {string.Join(", ", entry.Value)}");
                
                // 只显示前100个
                if (count >= 100)
                    break;
            }
        }
    }
} 